#!/usr/local/bin/python
#-*- encoding:utf-8 -*- 
import string 
from whoosh.index import create_in  
from whoosh.fields import *  
import jieba
import jieba.analyse
from whoosh.index import open_dir
from whoosh.scoring import *
from whoosh.matching import *
from whoosh.qparser import QueryParser
import utils


'''This script transform the test set to vectors, for matching model to score'''

config = utils.get_config();
analyzer = jieba.analyse.ChineseAnalyzer();

f = formats.Frequency();
schema = Schema(appid = ID(stored=True), content=TEXT(stored=True, analyzer=analyzer,vector=f), description=TEXT(stored=True, analyzer=analyzer));  
ix = open_dir(config.get('bm25', 'BackupIndex'));
reader = ix.reader();
searcher = ix.searcher();

parser = QueryParser('content', schema);
weightingM = BM25F();
dictMap = {};
d = open(config.get('rmls', 'dictionary'), 'r');
index = 0;
for line in d:
    dictMap[line.strip('\n').split('\t')[0].decode('utf-8')] = index;
    index += 1;
d.close();
MAX = len(dictMap);
print MAX;

def getQueryFeature(queryString):
    di = {}
    tli = analyzer(queryString.decode('utf-8'));
    for t in tli:
        term = t.text;
        if di.has_key(term):
            di[term] += 1;
        else:
            di[term] = 1;
    return di;

def getDocumentFeature(appID):
    di = {}
    docnum = searcher.document_number(appid= appID);
    if not docnum:
        return None;
    if not reader.has_vector(docnum, 'content'):
        return None;
    vec = reader.vector_as('frequency', docnum, 'content');
    for v in vec:
        term = v[0];
        tf = int(v[1]);
        score = weightingM.score(searcher, 'content', term, docnum, tf);
        di[term] = score
    return di;  

def format(di):
    if di == None:
        return None;
    if len(di.items()) == 0:
        return None;
    formatString = '';
    for key, value in di.items():
        print key;
        if dictMap.has_key(key):
            print 'XXXX'
            k = dictMap[key];
            formatString = '%s %s:%f' %(formatString, k, value);
    #formatString = '%s %d:%f' % (formatString, MAX+1, click);
    return formatString;


results = open('LabeledData/label_month.txt', 'r');
testingFile = open('TestSet/test.month.set', 'w');
testingFile.write('*\t%d\t%d\n' %(MAX, MAX));
count = 1;
for line in results:
    li = line.strip().split('|');
    queryID = str(count)
    count += 1;
    freq = int(li[0])
    queryString = li[1];
    for i in range(2, len(li)):
        appli = li[i].split(':');
        appID = appli[1];
        labeled = float(appli[2]);
        queryFeature = format(getQueryFeature(queryString));
        documentFeature = format(getDocumentFeature(appID));
        testingFile.write('%s,%s,%s,%s,%f,%d\n'%(queryID, appID, queryFeature, documentFeature, labeled, freq))


results.close();
testingFile.close();
